// main.cpp : Defines the entry point for the console application.
//

#include <string>
#include <iostream>
#include <fstream>
#include <sstream>
#include <algorithm>
#include <iterator>
#define _MY_STL_MACROS
#include <stlhelper.h>
#include <assert.h>
#include <deque>
#include <boost/tuple/tuple.hpp>
#include <boost/smart_ptr.hpp>


#include <print_error.h>
#include <libpq++/libpq++.h>
#include <postgres/pg_db_cursor.h>
#include <panda/panda.h>
#include <progress_indicator.h>
#include <portable_timer.h>
#include <perlutil.h>
#include <digest.h>
#include <panda/transfer_data_to_postgres.h>
#include <postgres/pg_copy_guard.h>
#include "get_arg.h"


using std::vector;
using std::sort;
using std::cout;
using std::cerr;
using std::string;
using std::ostream;
using boost::tie;
using boost::tuple;
using boost::make_tuple;




//AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

//	add_sequence_linked_data

//AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
struct t_digest_pi
{
	t_digest	digest;
	t_pi		pi;
	unsigned	mapped;
	t_digest_pi (unsigned d1, unsigned d2, unsigned d3, unsigned d4, unsigned pi_)
			: digest (d1, d2, d3, d4), pi (pi_), mapped (0){}
	t_digest_pi (){}
	bool operator <(const t_digest_pi& o) const {return digest < o.digest;}
	bool operator !=(const t_digest_pi& o) const {return digest != o.digest;}
	bool operator ==(const t_digest_pi& o) const {return digest == o.digest;}
};












//____________________________________________________________________________________

// 		Get seqhash to PI mapping from the PostgreSQL Panda database

//____________________________________________________________________________________
bool get_seqhash_to_pi_from_panda(	t_pg_cursor&		   	pg_db,
									vector<t_digest_pi>&	vec_digest_pi,
									ostream&				VERBOSE_STRM,
									ostream& 				CERR)
{

	// create binary cursor
	string SQL_cmd = "SELECT seq_hash[1], seq_hash[2], seq_hash[3], "
							"seq_hash[4], pi FROM sequences";
	if (!pg_db.Declare(SQL_cmd.c_str(), true))
		return pg_print_sql_err("create a binary cursor to retrieve sequence PIs",
									pg_db, SQL_cmd, CERR);


	{
		t_progress_indicator dots(VERBOSE_STRM, 1);

		// get data in batches
		unsigned seq_batch_size = 5000;
		vector<unsigned> hash;
		while (pg_db.Fetch(seq_batch_size))
		{
			int curr_batch_size = pg_db.Tuples();
			if (!curr_batch_size)
				break;
			dots++;

			for (int i = 0; i < curr_batch_size; ++i)

				vec_digest_pi.push_back(
							t_digest_pi(pg_db.getvalue<unsigned>(i, 0),
										pg_db.getvalue<unsigned>(i, 1),
										pg_db.getvalue<unsigned>(i, 2),
										pg_db.getvalue<unsigned>(i, 3),
										pg_db.getvalue<unsigned>(i, 4)));

		}
	}


	// close cursor
	if (!pg_db.Close())
		return pg_print_sql_err("close a binary cursor for retrieving sequence PIs",
									pg_db, "CLOSE panda_cursor", CERR);

	return true;
}







//____________________________________________________________________________________

//	parse_table_namespace

// 		Split table name into namespace and table

//____________________________________________________________________________________
tuple<string, string> parse_table_namespace(const string& table_name)
{
	string::size_type pos = table_name.find('.');
	vector<string> table_space;
	if (pos != string::npos)
		return make_tuple(string(table_name, 0, pos), string(table_name,  pos + 1));
	else
		return make_tuple("public", table_name);
}




//____________________________________________________________________________________

//	create_table

//		adds categories for pi, pi_reserved (for managing PIs in future)
//		and seq_hash. The later has the table name appended to hopefully make
//		the unique and prevent natural join from using it in joins
// 		returns true if the SQL table already exists

//____________________________________________________________________________________
bool create_table(t_pg_cursor& pg_db, const string& table_space, const string& table,
						const string& columns, ostream& CERR)
{
	string SQL_cmd = "CREATE TABLE " +
								table_space + "." + table		+
								"("
								" pi OID,"						+
								  table + "_pi_reserved OID, "	+
								  table + "_seq_hash OID[4], " 	+
								  columns						+
								");";

	if(!pg_db.ExecCommandOk(SQL_cmd.c_str()))
	{
		CERR		<< "ERROR!! PostgreSQL error\n"
					<< "Could not create the table '"
					<< (table_space + "." + table) << "' \n"
					<< pg_db.ErrorMessage()
					<< "SQL [ " << SQL_cmd << " ]\n";
		return false;
	}

	return true;
}








//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888

//		Main

//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
int main(int argc, char* argv[])
{
	// uncomment to time
	// t_portable_timer timer;

	// don't use C streams
	std::ios::sync_with_stdio(false);


	// we use cout to print progress so flush per call
	cout.setf(std::ios_base::unitbuf);

	// get command line arguments
	t_program_args args;
	try
	{
	if (!process_command_line_options(argc, argv, args))
		return 1;

	//____________________________________________________________________________________
	// 		Connect to PostgreSQL
	//____________________________________________________________________________________
	t_pg_cursor pg_db(args.db_login.c_str(), "panda_cursor");
	if ( pg_db.ConnectionBad() )
	{
		args.CERR()			<< "\nERROR!! Connection to database ["
							<< args.db_login << "] failed.\n"
							<< pg_db.ErrorMessage() << "\n\n";
		return 1;
	}

	//____________________________________________________________________________________
	// 		get namespace of table											 p
	//____________________________________________________________________________________
	string table, table_space;
	tie(table_space, table) = parse_table_namespace(args.table);

	//____________________________________________________________________________________
	// 		create table if necessary
	//____________________________________________________________________________________
	if (!pg_db.table_exists(table_space, table, args.CERR()))
	{
		if (!create_table(pg_db, table_space, table, args.columns, args.CERR()))
			return 1;
		args.VERBOSE() << "\tCreated table '" << args.table << "'\n";
	}


	//____________________________________________________________________________________
	// 		Get PIs mapping for all sequences from postgreSQL
	//____________________________________________________________________________________
	args.VERBOSE() << "\tReading the 128-bit hash to 32-bit PI mapping "
						"from the Panda database.\n";
	vector<t_digest_pi>	vec_digest_pi;
	if (!get_seqhash_to_pi_from_panda(pg_db, vec_digest_pi, args.VERBOSE(),
																	args.CERR()))
		return 1;

	args.VERBOSE() << "\tSorting the MD4 128-bit hash to Panda Identifier map...\n";
	sort(beg_to_end(vec_digest_pi));




	//____________________________________________________________________________________
	// 		Remove current entries before adding new ones
	//____________________________________________________________________________________
	if (args.remove)
	{
		args.VERBOSE()	<< "\tRemoving all items from the '"
						<< args.table << "' table first...\n";

		string SQL_cmd = "DELETE FROM " + args.table + ";";
        if(!pg_db.ExecCommandOk(SQL_cmd.c_str()))
		{
			args.CERR()	<< "\nERROR!!\tFailed to delete items in '"
						<< args.table << "'.\n"
						<< pg_db.ErrorMessage() << "\n\n";
			return 1;
		}

	}

	//____________________________________________________________________________________
	// 		Mapping entries to PIs
	//____________________________________________________________________________________
	{
		args.VERBOSE() << "\tEntering new items into the Panda PostgreSQL database ...\n";

		//
		// copy buffer management
		//
		t_pg_copy_guard copy_buffer(pg_db, args.table, args.CERR());
		if (!copy_buffer)
			return 1;


		unsigned cnt_matches = 0;
		unsigned cnt_no_matches = 0;
		vector<t_digest> vec_no_matches;

		{
			t_progress_indicator dots(args.VERBOSE(), 1000);
			t_digest_pi test_digest;
			t_digest_pi prev_digest;
			string		rest_of_line;

			// for each entry
			while (args.INPUT())
			{
				++dots;
				args.INPUT()
					>> test_digest.digest.digest[0]
					>> test_digest.digest.digest[1]
					>> test_digest.digest.digest[2]
					>> test_digest.digest.digest[3];
				if (!args.INPUT())
					break;

				// save rest of line
				getline(args.INPUT(),  rest_of_line);

				// lookup pi 32-bit from 128-bit hash and write to postgresql
				vector<t_digest_pi>::iterator find_beg, find_end;
				tie(find_beg, find_end) =
									equal_range(beg_to_end(vec_digest_pi), test_digest);
				if (find_beg != find_end)
				{
					cnt_matches++;
					find_beg->mapped = 1;
					ultoa(find_beg->pi, copy_buffer.buf) += "\t0\t";
					copy_buffer.buf += test_digest.digest;
					copy_buffer += rest_of_line + "\n";
					copy_buffer.cached_write();

				}
				else
				{
					// this 128-bit hash/sequence was not found in the database
					cnt_no_matches++;
					if (prev_digest != test_digest)
						vec_no_matches.push_back(test_digest.digest);
					prev_digest = test_digest;
				}

			}

			// flush buffer to postgresql
			if (!copy_buffer.end_copy())
				return false;
		}
		args.VERBOSE() << "\tFinished mapping ...\n\n";

	//____________________________________________________________________________________
	// 		gather statistics for mapping exercise
	//____________________________________________________________________________________
		// matching entries
		unsigned cnt_matching_PIs = 0;
		for (unsigned i = 0; i < vec_digest_pi.size(); ++i)
			cnt_matching_PIs += vec_digest_pi[i].mapped;
		args.VERBOSE()	<< "\t" << cnt_matches << "\tentries with hashes mapped to "
						<< cnt_matching_PIs << " PIs\n";

		// free memory
		vector<t_digest_pi>().swap(vec_digest_pi);


		// unmatched entries
		sort(beg_to_end(vec_no_matches));
		remove_duplicates(vec_no_matches);
		if (cnt_no_matches)
			args.VERBOSE()	<< "\t" << cnt_no_matches << "\tlines with "
							<< vec_no_matches.size()
							<< " hashes could not be mapped to PIs\n\n";
	}

	//____________________________________________________________________________________
	// 		PostgreSQL re-collecting database statistics.
	// 			commit transaction and reanalyse database
	//____________________________________________________________________________________
	{
		// commit transaction
		pg_db.Commit();

		if (args.analyse &&
			!pg_db.ExecCommandOk(("ANALYZE " + args.table).c_str()))
			args.CERR()	<< "ERROR!!\t"
						   "Failed to clean and analyse the new database tables\n"
						<< pg_db.ErrorMessage();
	}

	}
	catch (std::exception& e)
	{
		std_print_error(args.CERR(), e.what());
		return 1;
	}





	//
	//	Finished!!!
	//
	args.VERBOSE()		 << "\tFinished placing new data into the '" << args.table << "' table\n";

	return 0;
}


